#!/usr/bin/env python
# coding=utf8

import yaml
import urllib
import urllib2
import sqlite3
import csv
import xlrd
import datetime
import calendar
import httplib
import urlparse
import re
import threading
import string
import cookielib
import time
import random
import json

#from pytesser import *
#图形验证码
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.keys import Keys

from threading import Thread
from time import gmtime,strftime,time,sleep,ctime,clock
from BeautifulSoup import BeautifulSoup, SoupStrainer
from Queue import Queue

class QqDigger:
    config = yaml.load(open('config.yaml'))
    conn = sqlite3.connect(config['development']['database'])

    cursor = conn.cursor()
    
    table = {
        'shuoshuo':'shuoshuo',
        'urls':'urls',
        'reflist':'reflist',
        'agents':'agents'
        }
    
    today = datetime.date.today().strftime("%Y%m%d")
    
    url_header = config['qq']['shuoshuo']
    url_footer = "&ftype=0&sort=0&num=20&pos=0"
    loginUrl = config['qq']['login']
    
    urls = Queue()
    crawled_urls = []
    failed_urls = []
    failed_max = 1
    crawled_num  = 0
    counter = 0
    delay = 0.1
    run_start = clock()
    run_finish = clock()
    browser = webdriver.Ie()
    
    agentName = '2683406287'
    agentPwd = 'letmebe'
    
    def __init__(self, delay=0.1, retry = 3):
        self.run_start = clock()
        print "************* Update Digger Database At " + str(self.today) + " *************"
        self.delay = delay
        self.failed_max = retry
        self.initDatabase()
        self.autoLogin()        
        
    def initDatabase(self):
        try:
            self.cursor.execute(
                """
                CREATE TABLE IF NOT EXISTS %s(
                qq varchar(50),
                url varchar(255),
                visited boolean
                )
                """
                %(self.table['urls'])
                )
            self.cursor.execute(
                """
                CREATE TABLE IF NOT EXISTS %s(
                username varchar(50),
                password varchar(50)
                )
                """
                %(self.table['agents'])
                )
            self.cursor.execute(
                """
                CREATE TABLE IF NOT EXISTS %s(
                shuoshuo_id varchar(100),
                parent_id varchar(100),
                qq varchar(50),
                nickname varchar(100),
                content varchar(140),
                create_time datetime,
                crawled_time datetime
                )          
                """
                %(self.table['shuoshuo'])
                )

            self.cursor.execute(
                """
                CREATE TABLE IF NOT EXISTS %s(
                qq varchar(50),
                realname varchar(100),
                nickname varchar(100),
                question varchar(100),
                answer varchar(100),
                birthday datetime,
                totals integer 
                )          
                """
                %(self.table['reflist'])
                )
            """
            Commit
            """
            self.conn.commit()            
            print "************* Init Database Finished *************"
            
            self.saveWantedUrls()
            
        except Exception,e:
            print e
            
    def getAgent(self):
        try:
            pass
        except Exception,e:
            print e  
            
    def autoLogin(self):
        try:          
            self.browser.get(self.loginUrl)
            username = self.browser.find_element_by_id("u")
            username.clear()
            username.send_keys(self.agentName)            
            password = self.browser.find_element_by_id("p")
            password.clear()
            password.send_keys(self.agentPwd)            
            loginBtn = self.browser.find_element_by_id("subbtn")
            loginBtn.click()
            sleep(3)
            print "************* Login Finished *************"            
        except Exception,e:
            print e   
            
    def saveWantedUrls(self):
        try:
            while 0 == self.getRecNum('urls'):
                self.cursor.execute(
                """
                SELECT qq from %s
                """
                %('reflist')
                )
                qqs = self.cursor.fetchall()
                if None !=  qqs:
                    for q in qqs:
                        qq = q[0]
                        url = self.url_header + str(qq) + self.url_footer
                        self.urls.put(url)
                        self.cursor.execute(  
                        """
                        INSERT INTO urls(qq,url,visited) VALUES (?,?,?)
                        """            
                        ,(qq,url,'0')
                        ) 
                self.conn.commit()
                return True
        except Exception,e:
            print e
            
            
    def getWantedUrls(self):
        try:
            num = self.getRecNum('urls')
            if 0 != num:
                self.cursor.execute(
                """
                SELECT * from %s WHERE visited = '0'
                """
                %(self.table['urls'])
                )
                urls = self.cursor.fetchall()
                if None != urls:
                    for u in urls:
                        self.urls.put(u)
                    print "************* " + str(self.urls.qsize()) + " urls will be crawled *************"
                    return True
                else:
                    return False
                
            else:
                print "************* No new urls need be crawled *************"
                return False           
            
        except Exception,e:
            print e
            
    def getRecNum(self,table):
        try:
            self.cursor.execute(            
            """
            SELECT count(*) FROM %s
            """
            %(table)
            )
            return self.cursor.fetchone()[0]
        except Exception,e:
            print e
            
    def httpExists(self,url):
        host,path = urlparse.urlsplit(url)[1:3]
        if ':' in host:
            host,port =host.split(':',1)
            try:
                port = int(port)
            except ValueError:
                print 'Invalid port number %r' % (port,)
                return False
        else:
            port = None
            
        try:
            connection = httplib.HTTPConnection(host,port=port)
            connection.request("HEAD",path)
            resp = connection.getresponse()
            if resp.status == 200 or 301 or 400:
                found = True
            elif resp.status == 302 or 404:
                found = self.__httpExists__(urlparse.urljoin(url,resp.getheader('location','')))
            else:
                print "Status %d %s: %s" % (resp.status,resp.reason,url)
                found = False
        except Exception,e:
            print e.__class__,e,url
            found = False
        return found
    
    def getNewUrlsNum(self):
        try:
            self.cursor.execute(  
            """
            SELECT COUNT(*) FROM %s WHERE visited = '0'
            """            
            %(self.table['urls'])
            )
            return self.cursor.fetchone()[0]
        except Exception,e:
               print e
               
    def getShuoShuo(self):
        try:
            args = self.urls.get()
            url = args[1]
            qq = args[0]
            url_is_exists = self.httpExists(url)
            if url_is_exists:
                self.browser.get(url)
                try:
                    page = self.browser.page_source
                    self.dig(page)
                except Exception,e:
                    print "Information of "  + qq + " can not been got."
                    print e  
            else:
                print "************* The Url Is Not Exist! *************"
        except Exception,e:
                print e
                
    def dig(self,page):
        try:
            soup = BeautifulSoup(page)
            totals = soup.find(id='user_mood_count').text            
            print totals
            shuoshuo = soup.findAll(id=re.compile('^t_c_'))
            for s in shuoshuo:
                print s['id']
                print s.text
            #metadata = json.loads("AAA")
        except Exception,e:
                print e
            
            
    def crawl(self,threads = 1000):
        """
        Process finace infomation
        """
        try:
            if self.getNewUrlsNum() < threads:
                threads = self.getNewUrlsNum()
            else:
                pass
            pool = []            
            for index in xrange(0, threads):                
                spider = threading.Thread(target=self.getShuoShuo)
                spider.daemon = True
                pool.append(spider)
                
            for spider in pool:    
                spider.start()
                 
            for spider in pool:
                spider.join()          
            
        except Exception,e:
               print e
    
    def run(self):
        try:           
            while 0 != self.getNewUrlsNum():                
                self.getWantedUrls()
                self.crawl()
                # self.delCrawledUrls()
                # self.save()
            if 0 == self.getNewUrlsNum():                
                self.conn.close()
                self.run_finish = clock()
                print "************* Run Time: " + str(self.run_finish - self.run_start) + " *************"
                print "************* All Done! *************"
            else:
                pass
        except Exception,e:
            print e
    